import re


import scrapy
from spidertools.utils.time_utils import get_current_date

from commonresources.spider_items.hubei.items import HuBeiShengGongGongZiYuanJiaoYiPingTaiItem
from commonresources.spiders.basespider import BaseSpider


class HuBeiShengGongGongZiYuanJiaoYiPingTaiSpider(BaseSpider):
    """
        湖北省公共资源交易平台     https://www.hbggzyfwpt.cn/
    """
    name = 'HuBeiShengGongGongZiYuanJiaoYiPingTai'
    name_zh = "湖北省公共资源交易平台"
    start_urls = ['https://www.hbggzyfwpt.cn/jyxx/jsgcXmxx']

    def __init__(self, full_dose=False):
        self.num = 0
        self.areas = {}
        self.work = []
        super(HuBeiShengGongGongZiYuanJiaoYiPingTaiSpider, self).__init__(full_dose)

    def parse(self, response):
        business_types = response.xpath('//p[@id="businessType"]/a')
        areas = response.xpath('//select[@id="select"]/option')
        for area in areas:
            area_id = area.xpath('./@id').extract_first()
            if area_id == "000":
                continue
            area_zh = area.xpath('./text()').extract_first()
            self.areas[area_id] = area_zh
        print(self.areas)

        for business_type in business_types:
            href_part = business_type.xpath('./@href').extract_first()
            if "javascript" in href_part:
                continue
            business_type_zh = business_type.xpath('./text()').extract_first()
            yield scrapy.Request(
                url="https://www.hbggzyfwpt.cn" + href_part,
                meta={
                    "业务类型": business_type_zh,
                },
                callback=self.handle_response,
            )

    def fake_headers(self):
        return {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Cache-Control": "max-age=0",
            "Connection": "keep-alive",
            # "Content-Length": "147",
            "Content-Type": "application/x-www-form-urlencoded",
            # "Cookie": "JSESSIONID=a41c5211-8a59-4d69-a221-9127241fa346",
            "DNT": "1",
            "Host": "www.hbggzyfwpt.cn",
            "Origin": "https://www.hbggzyfwpt.cn",
            # "Referer": "https://www.hbggzyfwpt.cn/jyxx/tdsyq/crgg",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3823.400 QQBrowser/10.7.4307.400",
        }

    def fake_post_data(self,
                       currentPage,
                       area,
                       type="",
                       projectType="",
                       cqTypeValue="",
                       purchaserMode="",
                       industriesTypeCode=""):
        return {
            "currentPage": f"{currentPage}",
            "area": f"{area}",
            "type": f"{type}",
            "projectType": f"{projectType}",
            "cqTypeValue": f"{cqTypeValue}",
            "purchaserMode": f"{purchaserMode}",
            "purchaserModeType": f"{purchaserMode if purchaserMode != '9' else '7'}",
            "industriesTypeCode": f"{industriesTypeCode}",
            # "title": "",
            # "scrollValue": "",
            # "landProjectName": "",
            "publishTimeType": "4",
            "publishTimeStart": "2000-01-01",
            "publishTimeEnd": f"{get_current_date()}",
        }

    def handle_response(self, response):
        message_types = response.xpath('//p[@id="messageType"]/a')
        for area_code in self.areas:
            for message_type in message_types:
                href_part = message_type.xpath('./@href').extract_first()
                if "javascript" in href_part:  # 建设中
                    continue
                message_type_zh = message_type.xpath('./text()').extract_first()
                if response.meta['业务类型'] not in ['工程建设', "政府采购", '国有产权', '矿业权出让']:
                    option_list = [response.url, {"业务类型": response.meta['业务类型']},
                                   {"目标网站": "https://www.hbggzyfwpt.cn" + href_part},
                                   {"信息类型": message_type_zh}]
                    self.work.append(option_list)  # 行业
                    formdata = self.fake_post_data(1, area_code)
                    yield scrapy.FormRequest(
                        url="https://www.hbggzyfwpt.cn" + href_part,
                        callback=self.handle_response_response,
                        headers=self.fake_headers(),
                        dont_filter=True,
                        formdata=formdata,
                        meta={
                            "options_dict": {
                                "业务类型": response.meta['业务类型'],
                                "信息类型": message_type_zh,
                                "area_code": area_code,
                                "area": self.areas[area_code],
                            },
                            "page": 1,
                            "need_break": False,
                            "page_count": -1,
                        },
                    )
                if response.meta['业务类型'] == '工程建设':
                    industries = response.xpath('//div[@class="contSearchCon"]/div/ul/li')
                    if message_type_zh == "开标记录":
                        self.work.append(
                            [response.url, {"业务类型": response.meta['业务类型']},
                             {"目标网站": "https://www.hbggzyfwpt.cn" + href_part},
                             {"信息类型": message_type_zh}])
                        formdata = self.fake_post_data(1, area_code)
                        yield scrapy.FormRequest(
                            url="https://www.hbggzyfwpt.cn" + href_part,
                            callback=self.handle_response_response,
                            dont_filter=True,
                            headers=self.fake_headers(),
                            formdata=formdata,
                            meta={
                                "options_dict": {
                                    "业务类型": response.meta['业务类型'],
                                    "信息类型": message_type_zh,
                                    "area_code": area_code,
                                    "area": self.areas[area_code],
                                },
                                "page": 1,
                                "need_break": False,
                                "page_count": -1,
                            },
                        )
                    else:
                        for industry in industries:
                            industry_code = industry.xpath('./input/@value').extract_first()
                            industry_zh = industry.xpath('./a/text()').extract_first()
                            if industry_zh == "全部":
                                continue
                            self.work.append(
                                [response.url, {"业务类型": response.meta['业务类型']},
                                 {"目标网站": "https://www.hbggzyfwpt.cn" + href_part},
                                 {"信息类型": message_type_zh},
                                 {"industriesTypeCode": industry_code},
                                 {"industriesTypeCode_行业": industry_zh}])
                            formdata = self.fake_post_data(1, area_code, industriesTypeCode=industry_code)
                            yield scrapy.FormRequest(
                                url="https://www.hbggzyfwpt.cn" + href_part,
                                callback=self.handle_response_response,
                                dont_filter=True,
                                headers=self.fake_headers(),
                                formdata=formdata,
                                meta={
                                    "options_dict": {
                                        "industriesTypeCode": industry_code,
                                        "industriesTypeCode_行业": industry_zh,

                                        "业务类型": response.meta['业务类型'],
                                        "信息类型": message_type_zh,
                                        "area_code": area_code,
                                        "area": self.areas[area_code],
                                    },
                                    "page": 1,
                                    "need_break": False,
                                    "page_count": -1,
                                },
                            )
                elif response.meta['业务类型'] == '政府采购':
                    purchaser_modes = response.xpath('//p[@id="purchaserModeTypes"]/a')
                    for purchaser_mode in purchaser_modes:
                        purchaser_mode_code = purchaser_mode.xpath('./@onclick').extract_first().split(",")[-1][0:1]
                        purchaser_mode_zh = purchaser_mode.xpath('./text()').extract_first()
                        if purchaser_mode_zh == "全部":
                            continue
                        self.work.append(
                            [response.url, {"业务类型": response.meta['业务类型']},
                             {"目标网站": "https://www.hbggzyfwpt.cn" + href_part},
                             {"信息类型": message_type_zh},  # 采购方式
                             {"purchaserMode": purchaser_mode_code},
                             {"采购方式": purchaser_mode_zh}])

                        formdata = self.fake_post_data(1, area_code, purchaserMode=purchaser_mode_code)
                        yield scrapy.FormRequest(
                            url="https://www.hbggzyfwpt.cn" + href_part,
                            callback=self.handle_response_response,
                            dont_filter=True,
                            headers=self.fake_headers(),
                            formdata=formdata,
                            meta={
                                "options_dict": {
                                    "purchaserMode": purchaser_mode_code,
                                    "purchaserMode_采购方式": purchaser_mode_zh,

                                    "业务类型": response.meta['业务类型'],
                                    "信息类型": message_type_zh,
                                    "area_code": area_code,
                                    "area": self.areas[area_code],
                                },
                                "page": 1,
                                "need_break": False,
                                "page_count": -1,
                            },
                        )


                elif response.meta['业务类型'] == '国有产权':
                    project_types = response.xpath('//p[@id="cqType"]/a')
                    for project_type in project_types:
                        project_type_code = project_type.xpath('./@onclick').extract_first().split("'")[1]
                        project_type_zh = project_type.xpath('./text()').extract_first()

                        cqTypeValue_code = project_type.xpath('./@onclick').extract_first().split("(")[-1][0:1]
                        if project_type_zh == "全部":
                            continue
                        self.work.append(
                            [response.url, {"业务类型": response.meta['业务类型']},
                             {"目标网站": "https://www.hbggzyfwpt.cn" + href_part},
                             {"信息类型": message_type_zh},  # 产权类型
                             {"projectType": project_type_code},
                             {"cqTypeValue": cqTypeValue_code},
                             {"产权类型": project_type_zh}])

                        formdata = self.fake_post_data(1, area_code, cqTypeValue=cqTypeValue_code,
                                                       projectType=project_type_code)
                        yield scrapy.FormRequest(
                            url="https://www.hbggzyfwpt.cn" + href_part,
                            callback=self.handle_response_response,
                            headers=self.fake_headers(),
                            dont_filter=True,
                            formdata=formdata,
                            meta={
                                "options_dict": {
                                    "projectType": project_type_code,
                                    "cqTypeValue": cqTypeValue_code,
                                    "cqTypeValue_产权类型": project_type_zh,
                                    "业务类型": response.meta['业务类型'],
                                    "信息类型": message_type_zh,
                                    "area_code": area_code,
                                    "area": self.areas[area_code],
                                },
                                "page": 1,
                                "page_count": -1,
                                "need_break": False,
                            },
                        )

                elif response.meta['业务类型'] == '矿业权出让':
                    industries = response.xpath('//div[@class="contSearchCon"]/div/ul/li')
                    for industry in industries:
                        industry_href = industry.xpath('./a/@href').extract_first()
                        industry_type_code = industry_href.split('=')[1].split('&')[0]
                        industry_zh = industry.xpath('./a/text()').extract_first()  # 采矿权/探矿权

                        self.work.append(
                            [response.url, {"业务类型": response.meta['业务类型']},
                             {"目标网站": "https://www.hbggzyfwpt.cn" + href_part},
                             {"信息类型": message_type_zh},
                             {"type": industry_type_code},
                             {"行业": industry_zh}])

                        formdata = self.fake_post_data(1, area_code, type=industry_type_code)
                        yield scrapy.FormRequest(
                            url="https://www.hbggzyfwpt.cn" + href_part,
                            callback=self.handle_response_response,
                            dont_filter=True,
                            formdata=formdata,
                            meta={
                                "options_dict": {
                                    "type": industry_type_code,
                                    "type_行业": industry_zh,

                                    "业务类型": response.meta['业务类型'],
                                    "信息类型": message_type_zh,
                                    "area_code": area_code,
                                    "area": self.areas[area_code],
                                },
                                "page": 1,
                                "need_break": False,
                                "page_count": -1,
                            },
                        )

        if len(self.work) > 83 and not self.num:
            self.num += 1
            for w in self.work:
                print(w)

    def handle_response_response(self, response):
        try:
            if response.meta["page_count"] == -1:
                response.meta["page_count"] = int(re.findall(r"共(\d*?)页，当前", response.text)[0])
            objs = response.xpath('//div[@class="newListwenzi"]//table/tr')
            for obj in objs:
                item = dict()
                item['release_time'] = obj.xpath('./td[2]/text()').extract_first().strip()
                if not self.full_dose and item['release_time'] != get_current_date():
                    response.meta['need_break'] = True
                else:
                    item['source_platform'] = obj.xpath('./td[2]/a/font/text()').extract_first().strip()
                    item['source_platform_url'] = obj.xpath('./td[2]/a/@href').extract_first()
                    item['announcement_title'] = obj.xpath('./td[1]/a/text()').extract_first().strip()
                    item['origin_url'] = 'https://www.hbggzyfwpt.cn' + obj.xpath('./td[1]/a/@href').extract_first()
                    item['city'] = response.meta['options_dict']['area']
                    industry = "__行业（" + response.meta['options_dict']['industriesTypeCode_行业'] + "）" \
                        if "industriesTypeCode_行业" in response.meta['options_dict'] else ""

                    purchaserMode = "__采购方式（" + response.meta['options_dict']['purchaserMode_采购方式'] + "）" \
                        if "purchaserMode_采购方式" in response.meta['options_dict'] else ""

                    cqTypeValue = "__产权类型（" + response.meta['options_dict']['cqTypeValue_产权类型'] + "）" \
                        if "cqTypeValue_产权类型" in response.meta['options_dict'] else ""

                    type = "__行业（" + response.meta['options_dict']['type_行业'] + "）" \
                        if "type_行业" in response.meta['options_dict'] else ""

                    item['announcement_type'] = f"业务类型（{response.meta['options_dict']['业务类型']}）" \
                                                f"{industry}{purchaserMode}{cqTypeValue}{type}"
                    item['project_type'] = response.meta['options_dict']['信息类型']
                    item['item'] = HuBeiShengGongGongZiYuanJiaoYiPingTaiItem()
                    # print(item)
                    yield scrapy.Request(url=item['origin_url'],
                                         callback=self.parse_item,
                                         meta=item,
                                         dont_filter=True,
                                         )
            if not response.meta['need_break']:
                page = response.meta['page']
                page_count = response.meta['page_count']
                if page < page_count:
                    page += 1

                    industriesTypeCode = response.meta['options_dict']['industriesTypeCode'] \
                        if "industriesTypeCode" in response.meta['options_dict'] else ""

                    purchaserMode = response.meta['options_dict']['purchaserMode'] \
                        if "purchaserMode" in response.meta['options_dict'] else ""

                    cqTypeValue = response.meta['options_dict']['cqTypeValue'] \
                        if "cqTypeValue" in response.meta['options_dict'] else ""

                    type = response.meta['options_dict']['type'] \
                        if "type" in response.meta['options_dict'] else ""

                    formdata = self.fake_post_data(page,
                                                   response.meta['options_dict']['area_code'],
                                                   industriesTypeCode=industriesTypeCode,
                                                   purchaserMode=purchaserMode,
                                                   type=type,
                                                   cqTypeValue=cqTypeValue,
                                                   )
                    yield scrapy.FormRequest(
                        url=response.url,
                        callback=self.handle_response_response,
                        dont_filter=True,
                        formdata=formdata,
                        meta={
                            "options_dict": response.meta['options_dict'],
                            "page": page,
                            "need_break": False,
                            "page_count": page_count,
                        },
                    )

        except Exception as e:
            print(e)
            print(f'本项无内容，已停止该项 【{response.meta["options_dict"]["业务类型"]} 】的提取')
